<?php
/**
 * licenses : MIT
 * creatTime: 2020/5/18
 * author : weblinuxgame
 * filename: CrawlerService.php
 */

namespace App\Services;

use Closure;
use QL\QueryList;
use QL\Ext\Chrome;
use QL\Ext\CurlMulti;
use QL\Ext\AbsoluteUrl;
use Illuminate\Support\Facades\Log;
use Nesk\Puphpeteer\Resources\Browser;
use Tightenco\Collect\Support\Collection;

/**
 * 爬虫服务
 * Class CrawlerService
 * @package App\Services
 */
class CrawlerService extends BaseService
{
    protected $description = '爬虫启动命令';


    protected $options = [
        'timeout' => 6000,
        'ignoreHTTPSErrors' => true,
    ];

    const ALinkSelector = '//a';
    const ImageSelector = '//img';
    const NewsUrl = 'https://news.qq.com';
    const RegexpExpressSchema = '@regexp:';
    const ContentSelector = 'body > div.qq_conent.clearfix > div.LEFT';
    const ContentUrlRegexp = '@regexp:/\/([0-9]{8}(\w){4,100})(.html|.php)?$/';
    const TitleNewsUrlCollectionKey = 'titles';
    const OtherUrlCollectionKey = 'other';
    const UrlKey = 'url';
    const TitleKey = 'title';
    const ALinkUrlAttribute = 'href';

    protected $page;
    protected $browser;

    /**
     * 爬虫引擎
     * @return QueryList
     */
    public function engine()
    {
        $engine = new QueryList();
        // 注册插件，默认注册的方法名为: chrome
        $engine->use(Chrome::class);
        // 或者自定义注册的方法名
        $engine->use(Chrome::class, 'chrome');
        $engine->use(CurlMulti::class, 'curlMulti');
        $engine->use(AbsoluteUrl::class, 'absoluteUrl', 'absoluteUrlHelper');
        return $engine;
    }

    /**
     * 浏览器
     * @param string $url
     * @param array $options
     * @return QueryList
     */
    public function chrome(string $url, array $options = [])
    {
        return $this->engine()->chrome($url, array_merge($this->options, $options));
    }

    /**
     * 浏览器
     * @param Closure $handler
     * @param array $options
     * @return QueryList
     */
    public function page(Closure $handler, array $options = [])
    {
        $wrapper = function ($page, $browser) use ($handler) {
            $this->page = $page;
            $this->browser = $browser;
            $handler($page, $browser);
        };
        $wrapper = Closure::bind($wrapper, $this);
        return $this->engine()->chrome($wrapper, array_merge($this->options, $options));
    }

    /**
     * 全url
     * @param QueryList $engine
     * @param string $url
     * @return QueryList
     */
    protected function fullUrl(QueryList $engine, string $url = '')
    {
        if (empty($url)) {
            return $engine;
        }
        return $engine->absoluteUrl($url);
    }

    /**
     * 获取链接
     * @param string $url
     * @param string $selector
     * @return array
     */
    public function getLinks(string $url, string $selector = self::ALinkSelector)
    {
        $otherLinks = [];
        $engine = $this->chrome($url);
        $data = $this->fullUrl($engine, $url);
        $titleLinkCollection = new Collection();
        $elements = $data->find($selector)->getElements();
        // $isA = preg_match('/\b(a)\b/',$selector);
        foreach ($elements as $element) {
            $href = $element->getAttribute(self::ALinkUrlAttribute);
            if (empty($href) || !filter_var($href, FILTER_VALIDATE_URL)) {
                continue;
            }
            $title = $element->textContent;
            $title = preg_replace(["/\\n/", "/\\s/"], "", $title);
            if (empty($title) || is_numeric($title)) {
                $otherLinks[] = $href;
                continue;
            }
            $titleLinkCollection = $titleLinkCollection->push(self::createTitleItem($title, $href));
        }
        return [
            self::TitleNewsUrlCollectionKey => $titleLinkCollection->unique(),
            self::OtherUrlCollectionKey => array_unique($otherLinks),
        ];
    }

    /**
     * 创建
     * @param string $title
     * @param string $url
     * @return array
     */
    protected static function createTitleItem(string $title, string $url): array
    {
        return [self::TitleKey => $title, self::UrlKey => $url];
    }


    /**
     * 爬取QQ新闻
     */
    public function getQQNews()
    {
        $url = self::NewsUrl;
        $data = $this->getLinks($url);
        $titles = $data['titles'] ?? [];
        if (empty($titles) || !$titles instanceof Collection) {
            return;
        }
        $newsUrls = $this->getContentUrls($titles);
        if (empty($newsUrls) || $newsUrls->isEmpty()) {
            return;
        }
        foreach ($newsUrls as $item) {
            if (empty($item) || empty($item['url'])) {
                continue;
            }
            $content = $this->getContent($item['url']);
            $this->push(['title' => $item['title'], 'content' => $content, 'url' => $item['url']]);
        }
    }


    /**
     * 存储数据
     * @param array $data
     */
    public function push(array $data = [])
    {

    }

    /**
     * 获取文章内容
     * @param string $url
     * @param string $selector
     * @return array
     */
    public function getContent(string $url, $selector = self::ContentSelector): string
    {
        $dom = $this->get($url);
        if (empty($dom)) {
            return '';
        }
        $html = $dom->find($selector)->html();
        if (empty($elements)) {
            return '';
        }
        return $html;
    }

    /**
     * 获取
     * @param string $url
     * @return QueryList
     */
    public function get(string $url)
    {
        return $this->fullUrl($this->chrome($url), $url);
    }

    /**
     * 获取urls
     * @param Collection $collection
     * @param string $regexp
     * @return Collection|null
     */
    public function getContentUrls(Collection $collection, $regexp = self::ContentUrlRegexp): ?Collection
    {
        $data = null;
        if (false !== strpos($regexp, self::RegexpExpressSchema)) {
            $regexp = str_replace(self::RegexpExpressSchema, '', $regexp);
        }
        foreach ($collection as $item) {
            if (empty($item) || !is_array($item)) {
                continue;
            }
            $url = $item['url'] ?? '';
            $title = $item['title'] ?? '';
            if (empty($url) || empty($title)) {
                continue;
            }
            $match = preg_match($regexp, $url);
            if (0 === $match || false === $match) {
                continue;
            }
            if (empty($data)) {
                $data = new Collection();
            }
            $data = $data->push($item);
        }
        return $data;
    }

    /**
     * 获取 image urls
     * @param QueryList $dom
     * @param string $from
     * @param string $selector
     * @return Collection|null
     */
    public function getImages(QueryList $dom, $from = '', $selector = self::ImageSelector): ?Collection
    {
        $collections = new Collection();
        $elements = $dom->find($selector);
        foreach ($elements->attrs('src') as $item) {
            if (empty($item)) {
                continue;
            }
            if (!is_string($item) || empty($item)) {
                continue;
            }
            $url = $item;
            $collections->push(['image' => $url, 'from' => $from]);
        }
        return $collections->filter();
    }

    /**
     * 资源回收
     */
    public function __destruct()
    {
        $this->close();
        $this->page = null;
        $this->browser = null;
    }

    /**
     * 关闭
     */
    protected function close()
    {
        try {
            if (!empty($this->browser) && $this->browser instanceof Browser) {
                $this->browser->close();
            }
        } catch (Throwable $e) {
            Log::error($e->getMessage());
        }
    }

}
